# -*- coding: utf-8 -*-
'''
Created on Apr 3, 2013

@author: LONG HOANG GIANG
'''

from CrawlerLib2 import commonlib
import gzip as _gzip
import base64
import os
from Model.storymodelv3 import StoryModel

PATH = "/longhoanggiang/daudm/"

OUTPUT = "/longhoanggiang/truyentext/truyencotich/"

if not os.path.isdir(OUTPUT + "files/"):
    os.makedirs(OUTPUT + "files/", 0777)


data = []

def getDetail(path):
    fp = _gzip.open(path, 'rb')
    data = fp.read().replace('hindua88', '')
    fp.close()
    data = base64.decodestring(data)
    tree = commonlib.buildTreeFromXml(data)
    data = commonlib.Etree.tostring(tree.xpath("//body/*"))
    return data
    
def process():    
    
    smodel = StoryModel("/longhoanggiang/database/truyencotich")
    smodel.open(True)
    for idx in range(1, 7):
        filename = "listitem.xml" if idx == 1 else "listitem{0}.xml".format(idx)
        folder = "data" if idx == 1 else "data{0}".format(idx)
        data = commonlib.file_get_contents(PATH + filename)
        tree = commonlib.buildTreeFromXml(data)
        for item in tree.xpath("//item"):
            try:
                id = commonlib.stringify(item.xpath("./id"))
                name = commonlib.stringify(item.xpath("./name"))
                path = commonlib.stringify(item.xpath("./folder"))
                print id, name, path
                detail = getDetail(PATH + folder + "/" + path)
                smodel.add_story(name, detail, 0)
            except:
                pass
    smodel.close()
            
    
if __name__ == '__main__':
    
    process()
    print '> Finished'
    os._exit(1)
